/**
 * 爬取美团商圈数据工具
 */
const jsdom = require('jsdom');
const Crawler = require("crawler");
const fs = require('fs');
const http = require('http');
const path = require('path');
const querystring = require('querystring');
const { JSDOM } = jsdom;
const areaDataService = require('./areaDataService');
const { getCities, getAreaAndCities } = require('./citiesDataService');
const citiesFileName = path.join(process.cwd(), 'cities.json');
const config = require('./config');

function logger(msg) {
    console.log(msg);
}

/* 
* 方法:Array.remove(dx) 通过遍历,重构数组 
* 功能:删除数组元素. 
* 参数:dx删除元素的下标. 
*/
Array.prototype.remove = function (dx) {
    if (isNaN(dx) || dx > this.length) { return false; }
    for (var i = 0, n = 0; i < this.length; i++) {
        if (this[i] != this[dx]) {
            this[n++] = this[i]
        }
    }
    if (this.length >= 1)
        this.length -= 1
}

if (!config.cacheCityDatas) {
    fs.unlinkSync(citiesFileName);
}
if (!fs.existsSync(citiesFileName)) {

    let areaApi = 'http://' + config.apiHost + ':' + config.apiPort + '/api/area?level=2';

    logger.call(null, areaApi);

    http.get(areaApi, (res) => {

        if (res.statusCode === 200) {
            let rawData = '';

            res.on('data', (chunk) => rawData += chunk);
            res.on('end', () => {
                try {
                    const parsedData = JSON.parse(rawData);
                    console.log(parsedData.list);
                    if (parsedData.list) {
                        fs.writeFileSync(citiesFileName, JSON.stringify(parsedData.list));
                        console.log('write file sucessful!');
                    }
                    else {
                        console.log('write file err!');
                    }

                } catch (e) {
                    console.log(e.message);
                }
            });
        }
    });
}

var citiesMapping = new Map();
var urls = [];
var c = new Crawler({
    maxConnections: 10,
    // This will be called for each crawled page
    callback: function (error, res, done) {
        if (error) {
            console.log(error);
        } else {
            var $ = res.$;
            let {cities,citiesMap} = getCities($);
            citiesMapping = citiesMap;
            cities.forEach((item) => {
                urls.push(item.url);
            });
            //console.log(cities);
        }
        done();
    }
});

// Queue just one URL, with default callback
c.queue('http://www.dianping.com/');
// 存放每个城市对应的热门商圈地址url
let hotUrl = new Map();
/**
 * 
 * 
 * 'http://www.dianping.com/langfang' => 'http://www.dianping.com/shopall/33/0#BDBlock',
  'http://www.dianping.com/taiyuan' => 'http://www.dianping.com/shopall/35/0#BDBlock',
  'http://www.dianping.com/shijiazhuang' => 'http://www.dianping.com/shopall/24/0#BDBlock',
 * 
 * 
 * 
 */
var excuted = false;
//存放对应的城市区对应的商圈
var city_area = new Map();
//存放地铁对应的线路
var metro_line = new Map();
var all_data = new Map();
let urls2 = [];
var end = false;
c.on('drain', function () {
    console.log('drain');
    if (urls && urls.length > 0 && !end) {
        for (var index in urls) {
            this.queue([{
                uri: urls[index],
                jQuery: true,
                // The global callback won't be called
                callback: function (error, res, done) {
                    if (error) {
                        console.log(error);
                    } else {
                        var $ = res.$;
                        hotUrl.set(res.options.uri, 'http://' + path.join(config.spiderHost, $('.hot-title').eq(1).children('a').first().attr('href')));
                    }
                    done();
                    let dx = urls.indexOf(res.options.uri);
                    urls.remove(dx);
                    //console.log(urls.length);
                    if (urls.length === 0 && !excuted) {
                        //console.log(hotUrl);
                        excuted = true;
                        c.emit('drain');
                    }
                }
            }]);
        }
    }
    if (hotUrl && hotUrl.size > 0 && !end) {
        for (let [key, value] of hotUrl.entries()) {
            //console.log(value);
            urls2.push(value);
            this.queue([{
                uri: value,
                jQuery: true,
                // The global callback won't be called
                callback: function (error, res, done) {
                    if (error) {
                        console.log(error);
                    } else {
                        var $ = res.$;
                        //console.log(66666);
                        let data = Object.create(null);
                        //商圈
                        data.areas = [];
                        // 地铁线
                        data.metros = [];
                        $('.content_b').children('div').eq(1).children('dl').children('dt').children('a').each((i, element) => {
                            //区名称
                            let area = $(element).text();
                            let area_s = [];
                            $(element.parent.parent).children('dd').children('ul').children('li').children('a').each((i, element) => {
                                //console.log($(element).text());
                                //区对应的商圈
                                area_s.push($(element).text());
                            });
                            data.areas.push({ area_name: area, business_district_name: area_s });
                        }); 
                        //地铁线
                        $('.content_b').children('div').eq(3).children('dl').children('dt').children('a').each((i, element) => {
                            //地铁线名称
                            let line_name = $(element).text();
                            let line_stations = [];
                            $(element.parent.parent).children('dd').children('ul').children('li').children('a').each((i, element) => {
                                //console.log($(element).text());
                                //区对应的车站
                                line_stations.push($(element).text());
                            });
                            data.metros.push({ line_name: line_name, metro_near: line_stations });
                        }); 
                        all_data.set(key, data);
                    }
                    done();
                    let dx = urls2.indexOf(res.options.uri);
                    urls2.remove(dx);
                    if (urls2.length === 0 && !end) {                        
                        //console.log(JSON.stringify( all_data));
                        end = true;
                        // for (let [key, value] of all_data.entries()) {
                        //     console.log(key,value.areas);
                        // }
                        c.emit('drain');
                    }
                }
            }]);
        }
    }
    if (end) {
        let pushData = new Map();
        for (let [key, value] of all_data.entries()) {
            let {cityId} = citiesMapping.get(key);
            pushData.set(cityId, value);
        } 
        /**
         * 写入文件
         */
        let dataFilePath = path.join(process.cwd(), 'cities_data.json');
        if (!fs.existsSync(dataFilePath)) {
            
            var writeStream = fs.createWriteStream(dataFilePath);
            var json = '';
            for (let [key, value] of all_data.entries()) {
                let { cityId, cityName } = citiesMapping.get(key);
                let data = {
                    city_id: cityId,
                    city_uri: key,
                    city_name: cityName,
                    business_district: value.areas,
                    metro: value.metros
                };
                json = JSON.stringify(data);
                console.log(json);
                writeStream.write(json);
            }
            writeStream.end();
        }
        console.log('end !');
        //console.log(citiesMapping);       
        for (let [key, value] of pushData.entries()) {
            let data = {
                city_id: key,
                business_district: value.areas,
                metro: value.metros
            };
            //console.log(data);
            let data2 = {
                content: JSON.stringify(data)
            }; 
            //console.log(data2.content); 
            data2 = querystring.stringify(data2);   
            var opt = {
                protocol:'http:', 
                method: "POST",
                host: config.apiHost,
                port: config.apiPort,
                path: '/api/import/area',
                headers: {
                    "Content-Type": 'application/x-www-form-urlencoded',
                    "Content-Length": data2.length
                }
            };
            var req = http.request(opt, function (serverFeedback) {
                if (serverFeedback.statusCode == 200) {
                    var body = "";
                    serverFeedback.on('data', function (data) { 
                        body += data; 
                    }
                    ).on('end', function () {  
                        console.log('call php interface sucessed！', data2.toString());
                    });
                }
                else {
                    console.log('call php interface error', serverFeedback.statusCode.toString(),serverFeedback.statusMessage);
                }
            });
            req.write(data2 + "\n");
            req.end();  
            //console.log(all_data.size);
        }
    }
});
console.log('spider excuted');







